{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a989c965",
   "metadata": {},
   "source": [
    "# 课程目的：提高模型质量\n",
    "\n",
    "### 缺失值处理方法：\n",
    "###### 1.删除缺失值所在的列\n",
    "###### 2.填充\n",
    "###### 3.填充扩展"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "304e2704",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Rooms</th>\n",
       "      <th>Bathroom</th>\n",
       "      <th>Landsize</th>\n",
       "      <th>BuildingArea</th>\n",
       "      <th>YearBuilt</th>\n",
       "      <th>Lattitude</th>\n",
       "      <th>Longtitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>664</th>\n",
       "      <td>3</td>\n",
       "      <td>2.0</td>\n",
       "      <td>368.0</td>\n",
       "      <td>177.0</td>\n",
       "      <td>2009.0</td>\n",
       "      <td>-37.78460</td>\n",
       "      <td>145.09350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3270</th>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>586.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>1955.0</td>\n",
       "      <td>-37.74350</td>\n",
       "      <td>145.04860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3873</th>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>348.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-37.86720</td>\n",
       "      <td>145.04320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13170</th>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>521.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-37.63854</td>\n",
       "      <td>145.05179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1730</th>\n",
       "      <td>4</td>\n",
       "      <td>2.0</td>\n",
       "      <td>687.0</td>\n",
       "      <td>237.0</td>\n",
       "      <td>1983.0</td>\n",
       "      <td>-37.89310</td>\n",
       "      <td>145.04790</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Rooms  Bathroom  Landsize  BuildingArea  YearBuilt  Lattitude  \\\n",
       "664        3       2.0     368.0         177.0     2009.0  -37.78460   \n",
       "3270       2       1.0     586.0          80.0     1955.0  -37.74350   \n",
       "3873       2       1.0     348.0           NaN        NaN  -37.86720   \n",
       "13170      3       1.0     521.0           NaN        NaN  -37.63854   \n",
       "1730       4       2.0     687.0         237.0     1983.0  -37.89310   \n",
       "\n",
       "       Longtitude  \n",
       "664     145.09350  \n",
       "3270    145.04860  \n",
       "3873    145.04320  \n",
       "13170   145.05179  \n",
       "1730    145.04790  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# 自己定义的方法，用于比较处理缺失值的不同方法,内部用的是随机森林模型\n",
    "from score_dataset import score_dataset\n",
    "\n",
    "melb_data = pd.read_csv('../melb_data.csv')\n",
    "y = melb_data.Price\n",
    "X = melb_data[['Rooms', 'Bathroom', 'Landsize', 'BuildingArea', 'YearBuilt', 'Lattitude', 'Longtitude']]\n",
    "# 将数据分割为训练和验证数据，都有特征和预测目标值\n",
    "# 分割基于随机数生成器。为random_state参数提供一个数值可以保证每次得到相同的分割\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X, y, random_state = 0)\n",
    "# 用head()瞄一眼X__train\n",
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e02667ec",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAE from Approach 1 (Drop columns with missing values):\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "180860.37877504269"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1.删除缺失值所在的列\n",
    "# 获取缺少值的列名称\n",
    "cols_with_missing = [col for col in X_train.columns\n",
    "                     if X_train[col].isnull().any()]\n",
    "\n",
    "# 删除训练和验证数据中的列\n",
    "reduced_X_train = X_train.drop(cols_with_missing, axis=1)\n",
    "reduced_X_valid = X_valid.drop(cols_with_missing, axis=1)\n",
    "\n",
    "print(\"MAE from Approach 1 (Drop columns with missing values):\")\n",
    "score_dataset(reduced_X_train, reduced_X_valid, y_train, y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7ff38a15",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAE from Approach 2 (Imputation):\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "175016.09702559788"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2.我们使用SimpleImputer用每一列的平均值填充缺失的值\n",
    "from sklearn.impute import SimpleImputer\n",
    "\n",
    "# 填充\n",
    "my_imputer = SimpleImputer()\n",
    "imputed_X_train = pd.DataFrame(my_imputer.fit_transform(X_train))\n",
    "imputed_X_valid = pd.DataFrame(my_imputer.transform(X_valid))\n",
    "\n",
    "# 填充移除了列名;补回来\n",
    "imputed_X_train.columns = X_train.columns\n",
    "imputed_X_valid.columns = X_valid.columns\n",
    "\n",
    "print(\"MAE from Approach 2 (Imputation):\")\n",
    "score_dataset(imputed_X_train, imputed_X_valid, y_train, y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bd250b13",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAE from Approach 3 (An Extension to Imputation):\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "175181.65307310002"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 3.填充扩展,填充缺失的值，同时记录哪些值是填充的。\n",
    "# Make copy to avoid changing original data (when imputing)\n",
    "X_train_plus = X_train.copy()\n",
    "X_valid_plus = X_valid.copy()\n",
    "\n",
    "# Make new columns indicating what will be imputed\n",
    "for col in cols_with_missing:\n",
    "    X_train_plus[col + '_was_missing'] = X_train_plus[col].isnull()\n",
    "    X_valid_plus[col + '_was_missing'] = X_valid_plus[col].isnull()\n",
    "\n",
    "# Imputation\n",
    "my_imputer = SimpleImputer()\n",
    "imputed_X_train_plus = pd.DataFrame(my_imputer.fit_transform(X_train_plus))\n",
    "imputed_X_valid_plus = pd.DataFrame(my_imputer.transform(X_valid_plus))\n",
    "\n",
    "# Imputation removed column names; put them back\n",
    "imputed_X_train_plus.columns = X_train_plus.columns\n",
    "imputed_X_valid_plus.columns = X_valid_plus.columns\n",
    "\n",
    "print(\"MAE from Approach 3 (An Extension to Imputation):\")\n",
    "score_dataset(imputed_X_train_plus, imputed_X_valid_plus, y_train, y_valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e169087d",
   "metadata": {},
   "source": [
    "### 总结：\n",
    "一般来说，比起简单地删除具有缺失值的列(在方法1中)，填充缺失值(在方法2和方法3中)会得到更好的效果。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
